<!DOCTYPE html>
<html lang="zh-CN">

<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
  <meta name="theme-color" content="#222">
  <meta name="generator" content="Hexo 4.2.1">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/safari-pinned-tab.svg" color="#222">
  <link rel="stylesheet" href="/css/main.css">
  <link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">
  <link rel="stylesheet" href="/lib/pace/pace-theme-minimal.min.css">
  <script src="/lib/pace/pace.min.js"></script>
  <script id="hexo-configurations">
    var NexT = window.NexT ||
    {};
    var CONFIG = {
      "hostname": "cuiqingcai.com",
      "root": "/",
      "scheme": "Pisces",
      "version": "7.8.0",
      "exturl": false,
      "sidebar":
      {
        "position": "right",
        "width": 360,
        "display": "post",
        "padding": 18,
        "offset": 12,
        "onmobile": false,
        "widgets": [
          {
            "type": "image",
            "name": "阿布云",
            "enable": false,
            "url": "https://www.abuyun.com/http-proxy/introduce.html",
            "src": "https://qiniu.cuiqingcai.com/88au8.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "天验",
            "enable": true,
            "url": "https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850",
            "src": "https://qiniu.cuiqingcai.com/bco2a.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "华为云",
            "enable": false,
            "url": "https://activity.huaweicloud.com/2020_618_promotion/index.html?bpName=5f9f98a29e2c40b780c1793086f29fe2&bindType=1&salesID=wangyubei",
            "src": "https://qiniu.cuiqingcai.com/y42ik.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "张小鸡",
            "enable": false,
            "url": "http://www.zxiaoji.com/",
            "src": "https://qiniu.cuiqingcai.com/fm72f.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "Luminati",
            "src": "https://qiniu.cuiqingcai.com/ikkq9.jpg",
            "url": "https://luminati-china.io/?affiliate=ref_5fbbaaa9647883f5c6f77095",
            "width": "100%",
            "enable": false
      },
          {
            "type": "image",
            "name": "IPIDEA",
            "url": "http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc",
            "src": "https://qiniu.cuiqingcai.com/0ywun.png",
            "width": "100%",
            "enable": true
      },
          {
            "type": "tags",
            "name": "标签云",
            "enable": true
      },
          {
            "type": "categories",
            "name": "分类",
            "enable": true
      },
          {
            "type": "friends",
            "name": "友情链接",
            "enable": true
      },
          {
            "type": "hot",
            "name": "猜你喜欢",
            "enable": true
      }]
      },
      "copycode":
      {
        "enable": true,
        "show_result": true,
        "style": "mac"
      },
      "back2top":
      {
        "enable": true,
        "sidebar": false,
        "scrollpercent": true
      },
      "bookmark":
      {
        "enable": false,
        "color": "#222",
        "save": "auto"
      },
      "fancybox": false,
      "mediumzoom": false,
      "lazyload": false,
      "pangu": true,
      "comments":
      {
        "style": "tabs",
        "active": "gitalk",
        "storage": true,
        "lazyload": false,
        "nav": null,
        "activeClass": "gitalk"
      },
      "algolia":
      {
        "hits":
        {
          "per_page": 10
        },
        "labels":
        {
          "input_placeholder": "Search for Posts",
          "hits_empty": "We didn't find any results for the search: ${query}",
          "hits_stats": "${hits} results found in ${time} ms"
        }
      },
      "localsearch":
      {
        "enable": true,
        "trigger": "auto",
        "top_n_per_article": 10,
        "unescape": false,
        "preload": false
      },
      "motion":
      {
        "enable": false,
        "async": false,
        "transition":
        {
          "post_block": "bounceDownIn",
          "post_header": "slideDownIn",
          "post_body": "slideDownIn",
          "coll_header": "slideLeftIn",
          "sidebar": "slideUpIn"
        }
      },
      "path": "search.xml"
    };

  </script>
  <meta name="description" content="13.13 Scrapy 爬取新浪微博 前面讲解了 Scrapy 中各个模块基本使用方法以及代理池、Cookies 池。接下来我们以一个反爬比较强的网站新浪微博为例，来实现一下 Scrapy 的大规模爬取。 1. 本节目标 本次爬取的目标是新浪微博用户的公开基本信息，如用户昵称、头像、用户的关注、粉丝列表以及发布的微博等，这些信息抓取之后保存至 MongoDB。 2. 准备工作 请确保前文所讲的代">
  <meta property="og:type" content="article">
  <meta property="og:title" content="[Python3网络爬虫开发实战] 13.13–Scrapy 爬取新浪微博">
  <meta property="og:url" content="https://cuiqingcai.com/8453.html">
  <meta property="og:site_name" content="静觅">
  <meta property="og:description" content="13.13 Scrapy 爬取新浪微博 前面讲解了 Scrapy 中各个模块基本使用方法以及代理池、Cookies 池。接下来我们以一个反爬比较强的网站新浪微博为例，来实现一下 Scrapy 的大规模爬取。 1. 本节目标 本次爬取的目标是新浪微博用户的公开基本信息，如用户昵称、头像、用户的关注、粉丝列表以及发布的微博等，这些信息抓取之后保存至 MongoDB。 2. 准备工作 请确保前文所讲的代">
  <meta property="og:locale" content="zh_CN">
  <meta property="og:image" content="https://qiniu.cuiqingcai.com/2019-11-27-034836.jpg">
  <meta property="og:image" content="https://qiniu.cuiqingcai.com/2019-11-27-034839.jpg">
  <meta property="og:image" content="https://qiniu.cuiqingcai.com/2019-11-27-034852.png">
  <meta property="og:image" content="https://qiniu.cuiqingcai.com/2019-11-27-034901.jpg">
  <meta property="og:image" content="https://qiniu.cuiqingcai.com/2019-11-27-034906.jpg">
  <meta property="og:image" content="https://qiniu.cuiqingcai.com/2019-11-27-034931.jpg">
  <meta property="og:image" content="https://qiniu.cuiqingcai.com/2019-11-27-034947.jpg">
  <meta property="og:image" content="https://qiniu.cuiqingcai.com/2019-11-27-035001.jpg">
  <meta property="article:published_time" content="2019-12-08T01:50:48.000Z">
  <meta property="article:modified_time" content="2021-12-18T13:11:11.553Z">
  <meta property="article:author" content="崔庆才">
  <meta property="article:tag" content="崔庆才">
  <meta property="article:tag" content="静觅">
  <meta property="article:tag" content="PHP">
  <meta property="article:tag" content="Java">
  <meta property="article:tag" content="Python">
  <meta property="article:tag" content="Spider">
  <meta property="article:tag" content="爬虫">
  <meta property="article:tag" content="Web">
  <meta property="article:tag" content="Kubernetes">
  <meta property="article:tag" content="深度学习">
  <meta property="article:tag" content="机器学习">
  <meta property="article:tag" content="数据分析">
  <meta property="article:tag" content="网络">
  <meta property="article:tag" content="IT">
  <meta property="article:tag" content="技术">
  <meta property="article:tag" content="博客">
  <meta name="twitter:card" content="summary">
  <meta name="twitter:image" content="https://qiniu.cuiqingcai.com/2019-11-27-034836.jpg">
  <link rel="canonical" href="https://cuiqingcai.com/8453.html">
  <script id="page-configurations">
    // https://hexo.io/docs/variables.html
    CONFIG.page = {
      sidebar: "",
      isHome: false,
      isPost: true,
      lang: 'zh-CN'
    };

  </script>
  <title>[Python3网络爬虫开发实战] 13.13–Scrapy 爬取新浪微博 | 静觅</title>
  <meta name="google-site-verification" content="p_bIcnvirkFzG2dYKuNDivKD8-STet5W7D-01woA2fc" />
  <noscript>
    <style>
      .use-motion .brand,
      .use-motion .menu-item,
      .sidebar-inner,
      .use-motion .post-block,
      .use-motion .pagination,
      .use-motion .comments,
      .use-motion .post-header,
      .use-motion .post-body,
      .use-motion .collection-header
      {
        opacity: initial;
      }

      .use-motion .site-title,
      .use-motion .site-subtitle
      {
        opacity: initial;
        top: initial;
      }

      .use-motion .logo-line-before i
      {
        left: initial;
      }

      .use-motion .logo-line-after i
      {
        right: initial;
      }

    </style>
  </noscript>
  <link rel="alternate" href="/atom.xml" title="静觅" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container">
    <div class="headband"></div>
    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner">
        <div class="site-brand-container">
          <div class="site-nav-toggle">
            <div class="toggle" aria-label="切换导航栏">
              <span class="toggle-line toggle-line-first"></span>
              <span class="toggle-line toggle-line-middle"></span>
              <span class="toggle-line toggle-line-last"></span>
            </div>
          </div>
          <div class="site-meta">
            <a href="/" class="brand" rel="start">
              <span class="logo-line-before"><i></i></span>
              <h1 class="site-title">静觅 <span class="site-subtitle"> 崔庆才的个人站点 </span>
              </h1>
              <span class="logo-line-after"><i></i></span>
            </a>
          </div>
          <div class="site-nav-right">
            <div class="toggle popup-trigger">
              <i class="fa fa-search fa-fw fa-lg"></i>
            </div>
          </div>
        </div>
        <nav class="site-nav">
          <ul id="menu" class="main-menu menu">
            <li class="menu-item menu-item-home">
              <a href="/" rel="section">首页</a>
            </li>
            <li class="menu-item menu-item-archives">
              <a href="/archives/" rel="section">文章列表</a>
            </li>
            <li class="menu-item menu-item-tags">
              <a href="/tags/" rel="section">文章标签</a>
            </li>
            <li class="menu-item menu-item-categories">
              <a href="/categories/" rel="section">文章分类</a>
            </li>
            <li class="menu-item menu-item-about">
              <a href="/about/" rel="section">关于博主</a>
            </li>
            <li class="menu-item menu-item-message">
              <a href="/message/" rel="section">给我留言</a>
            </li>
            <li class="menu-item menu-item-search">
              <a role="button" class="popup-trigger">搜索 </a>
            </li>
          </ul>
        </nav>
        <div class="search-pop-overlay">
          <div class="popup search-popup">
            <div class="search-header">
              <span class="search-icon">
                <i class="fa fa-search"></i>
              </span>
              <div class="search-input-container">
                <input autocomplete="off" autocapitalize="off" placeholder="搜索..." spellcheck="false" type="search" class="search-input">
              </div>
              <span class="popup-btn-close">
                <i class="fa fa-times-circle"></i>
              </span>
            </div>
            <div id="search-result">
              <div id="no-result">
                <i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>
              </div>
            </div>
          </div>
        </div>
      </div>
    </header>
    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
      <span>0%</span>
    </div>
    <div class="reading-progress-bar"></div>
    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div class="content post posts-expand">
            <article itemscope itemtype="http://schema.org/Article" class="post-block single" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/8453.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h1 class="post-title" itemprop="name headline"> [Python3网络爬虫开发实战] 13.13–Scrapy 爬取新浪微博 </h1>
                <div class="post-meta">
                  <span class="post-meta-item">
                    <span class="post-meta-item-icon">
                      <i class="far fa-user"></i>
                    </span>
                    <span class="post-meta-item-text">作者</span>
                    <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                  </span>
                  <span class="post-meta-item">
                    <span class="post-meta-item-icon">
                      <i class="far fa-calendar"></i>
                    </span>
                    <span class="post-meta-item-text">发表于</span>
                    <time title="创建时间：2019-12-08 09:50:48" itemprop="dateCreated datePublished" datetime="2019-12-08T09:50:48+08:00">2019-12-08</time>
                  </span>
                  <span class="post-meta-item">
                    <span class="post-meta-item-icon">
                      <i class="far fa-folder"></i>
                    </span>
                    <span class="post-meta-item-text">分类于</span>
                    <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                      <a href="/categories/Python/" itemprop="url" rel="index"><span itemprop="name">Python</span></a>
                    </span>
                  </span>
                  <span id="/8453.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 13.13–Scrapy 爬取新浪微博" title="阅读次数">
                    <span class="post-meta-item-icon">
                      <i class="fa fa-eye"></i>
                    </span>
                    <span class="post-meta-item-text">阅读次数：</span>
                    <span class="leancloud-visitors-count"></span>
                  </span>
                  <span class="post-meta-item" title="本文字数">
                    <span class="post-meta-item-icon">
                      <i class="far fa-file-word"></i>
                    </span>
                    <span class="post-meta-item-text">本文字数：</span>
                    <span>17k</span>
                  </span>
                  <span class="post-meta-item" title="阅读时长">
                    <span class="post-meta-item-icon">
                      <i class="far fa-clock"></i>
                    </span>
                    <span class="post-meta-item-text">阅读时长 &asymp;</span>
                    <span>15 分钟</span>
                  </span>
                </div>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="advertisements">
                  <div class="item">
                    <a href="http://i0k.cn/4UUsd" target="_blank">
                      <img src="https://qiniu.cuiqingcai.com/dsdhf.jpg">
                    </a>
                  </div>
                </div>
                <h1 id="13-13-Scrapy-爬取新浪微博"><a href="#13-13-Scrapy-爬取新浪微博" class="headerlink" title="13.13 Scrapy 爬取新浪微博"></a>13.13 Scrapy 爬取新浪微博</h1>
                <p>前面讲解了 Scrapy 中各个模块基本使用方法以及代理池、Cookies 池。接下来我们以一个反爬比较强的网站新浪微博为例，来实现一下 Scrapy 的大规模爬取。</p>
                <h3 id="1-本节目标"><a href="#1-本节目标" class="headerlink" title="1. 本节目标"></a>1. 本节目标</h3>
                <p>本次爬取的目标是新浪微博用户的公开基本信息，如用户昵称、头像、用户的关注、粉丝列表以及发布的微博等，这些信息抓取之后保存至 MongoDB。</p>
                <h3 id="2-准备工作"><a href="#2-准备工作" class="headerlink" title="2. 准备工作"></a>2. 准备工作</h3>
                <p>请确保前文所讲的代理池、Cookies 池已经实现并可以正常运行，安装 Scrapy、PyMongo 库，如没有安装可以参考前文内容。</p>
                <h3 id="3-爬取思路"><a href="#3-爬取思路" class="headerlink" title="3. 爬取思路"></a>3. 爬取思路</h3>
                <p>首先我们要实现用户的大规模爬取。这里采用的爬取方式是，以微博的几个大 V 为起始点，爬取他们各自的粉丝和关注列表，然后获取粉丝和关注列表的粉丝和关注列表，以此类推，这样下去就可以实现递归爬取。如果一个用户与其他用户有社交网络上的关联，那他们的信息就会被爬虫抓取到，这样我们就可以做到对所有用户的爬取。通过这种方式，我们可以得到用户的唯一 ID，再根据 ID 获取每个用户发布的微博即可。</p>
                <h3 id="4-爬取分析"><a href="#4-爬取分析" class="headerlink" title="4. 爬取分析"></a>4. 爬取分析</h3>
                <p>这里我们选取的爬取站点是：<a href="https://m.weibo.cn，此站点是微博移动端的站点。打开该站点会跳转到登录页面，这是因为主页做了登录限制。不过我们可以直接打开某个用户详情页面，如图" target="_blank" rel="noopener">https://m.weibo.cn，此站点是微博移动端的站点。打开该站点会跳转到登录页面，这是因为主页做了登录限制。不过我们可以直接打开某个用户详情页面，如图</a> 13-32 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034836.jpg" alt=""> 图 13-32 个人详情页面 我们在页面最上方可以看到她的关注和粉丝数量。我们点击关注，进入到她的关注列表，如图 13-33 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034839.jpg" alt=""> 图 13-33 关注列表 我们打开开发者工具，切换到 XHR 过滤器，一直下拉关注列表，即可看到下方会出现很多 Ajax 请求，这些请求就是获取关注列表的 Ajax 请求，如图 13-34 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034852.png" alt=""> 图 13-34 请求列表 我们打开第一个 Ajax 请求看一下，发现它的链接为：<a href="https://m.weibo.cn/api/container/getIndex?containerid=231051_-_followers_-_1916655407&amp;luicode=10000011&amp;lfid=1005051916655407&amp;featurecode=20000320&amp;type=uid&amp;value=1916655407&amp;page=2" target="_blank" rel="noopener">https://m.weibo.cn/api/container/getIndex?containerid=231051<em>-_followers</em>-_1916655407&amp;luicode=10000011&amp;lfid=1005051916655407&amp;featurecode=20000320&amp;type=uid&amp;value=1916655407&amp;page=2</a>，详情如图 13-35 和 13-36 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034901.jpg" alt=""> 图 13-35 请求详情 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034906.jpg" alt=""> 图 13-36 响应结果 请求类型是 GET 类型，返回结果是 JSON 格式，我们将其展开之后即可看到其关注的用户的基本信息。接下来我们只需要构造这个请求的参数。此链接一共有 7 个参数，如图 13-37 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034931.jpg" alt=""> 图 13-37 参数信息 其中最主要的参数就是 containerid 和 page。有了这两个参数，我们同样可以获取请求结果。我们可以将接口精简为：<a href="https://m.weibo.cn/api/container/getIndex?containerid=231051_-_followers_-_1916655407&amp;page=2" target="_blank" rel="noopener">https://m.weibo.cn/api/container/getIndex?containerid=231051<em>-_followers</em>-_1916655407&amp;page=2</a>，这里的 containerid 的前半部分是固定的，后半部分是用户的 id。所以这里参数就可以构造出来了，只需要修改 containerid 最后的 id 和 page 参数即可获取分页形式的关注列表信息。 利用同样的方法，我们也可以分析用户详情的 Ajax 链接、用户微博列表的 Ajax 链接，如下所示：</p>
                <figure class="highlight ini">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="comment"># 用户详情 API</span></span><br><span class="line"><span class="attr">user_url</span> = <span class="string">'https://m.weibo.cn/api/container/getIndex?uid=&#123;uid&#125;&amp;type=uid&amp;value=&#123;uid&#125;&amp;containerid=100505&#123;uid&#125;'</span></span><br><span class="line"><span class="comment"># 关注列表 API</span></span><br><span class="line"><span class="attr">follow_url</span> = <span class="string">'https://m.weibo.cn/api/container/getIndex?containerid=231051_-_followers_-_&#123;uid&#125;&amp;page=&#123;page&#125;'</span></span><br><span class="line"><span class="comment"># 粉丝列表 API</span></span><br><span class="line"><span class="attr">fan_url</span> = <span class="string">'https://m.weibo.cn/api/container/getIndex?containerid=231051_-_fans_-_&#123;uid&#125;&amp;page=&#123;page&#125;'</span></span><br><span class="line"><span class="comment"># 微博列表 API</span></span><br><span class="line"><span class="attr">weibo_url</span> = <span class="string">'https://m.weibo.cn/api/container/getIndex?uid=&#123;uid&#125;&amp;type=uid&amp;page=&#123;page&#125;&amp;containerid=107603&#123;uid&#125;'</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>此处的 uid 和 page 分别代表用户 ID 和分页页码。 注意，这个 API 可能随着时间的变化或者微博的改版而变化，以实测为准。 我们从几个大 V 开始抓取，抓取他们的粉丝、关注列表、微博信息，然后递归抓取他们的粉丝和关注列表的粉丝、关注列表、微博信息，递归抓取，最后保存微博用户的基本信息、关注和粉丝列表、发布的微博。 我们选择 MongoDB 作为存储的数据库，可以更方便地存储用户的粉丝和关注列表。</p>
                <h3 id="5-新建项目"><a href="#5-新建项目" class="headerlink" title="5. 新建项目"></a>5. 新建项目</h3>
                <p>接下来，我们用 Scrapy 来实现这个抓取过程。首先创建一个项目，命令如下所示：</p>
                <figure class="highlight ebnf">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="attribute">scrapy startproject weibo</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>进入项目中，新建一个 Spider，名为 weibocn，命令如下所示：</p>
                <figure class="highlight css">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="selector-tag">scrapy</span> <span class="selector-tag">genspider</span> <span class="selector-tag">weibocn</span> <span class="selector-tag">m</span><span class="selector-class">.weibo</span><span class="selector-class">.cn</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>我们首先修改 Spider，配置各个 Ajax 的 URL，选取几个大 V，将他们的 ID 赋值成一个列表，实现 start_requests() 方法，也就是依次抓取各个大 V 的个人详情，然后用 parse_user() 进行解析，如下所示：</p>
                <figure class="highlight ruby">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">from scrapy import Request, Spider</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">WeiboSpider</span>(<span class="title">Spider</span>):</span></span><br><span class="line">    name = <span class="string">'weibocn'</span></span><br><span class="line">    allowed_domains = [<span class="string">'m.weibo.cn'</span>]</span><br><span class="line">    user_url = <span class="string">'https://m.weibo.cn/api/container/getIndex?uid=&#123;uid&#125;&amp;type=uid&amp;value=&#123;uid&#125;&amp;containerid=100505&#123;uid&#125;'</span></span><br><span class="line">    follow_url = <span class="string">'https://m.weibo.cn/api/container/getIndex?containerid=231051_-_followers_-_&#123;uid&#125;&amp;page=&#123;page&#125;'</span></span><br><span class="line">    fan_url = <span class="string">'https://m.weibo.cn/api/container/getIndex?containerid=231051_-_fans_-_&#123;uid&#125;&amp;page=&#123;page&#125;'</span></span><br><span class="line">    weibo_url = <span class="string">'https://m.weibo.cn/api/container/getIndex?uid=&#123;uid&#125;&amp;type=uid&amp;page=&#123;page&#125;&amp;containerid=107603&#123;uid&#125;'</span></span><br><span class="line">    start_users = [<span class="string">'3217179555'</span>, <span class="string">'1742566624'</span>, <span class="string">'2282991915'</span>, <span class="string">'1288739185'</span>, <span class="string">'3952070245'</span>, <span class="string">'5878659096'</span>]</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">start_requests</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">for</span> uid <span class="keyword">in</span> <span class="keyword">self</span>.<span class="symbol">start_users:</span></span><br><span class="line">            <span class="keyword">yield</span> Request(<span class="keyword">self</span>.user_url.format(uid=uid), callback=<span class="keyword">self</span>.parse_user)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">parse_user</span><span class="params">(<span class="keyword">self</span>, response)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.logger.debug(response)</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <h3 id="6-创建-Item"><a href="#6-创建-Item" class="headerlink" title="6. 创建 Item"></a>6. 创建 Item</h3>
                <p>接下来，我们解析用户的基本信息并生成 Item。这里我们先定义几个 Item，如用户、用户关系、微博的 Item，如下所示：</p>
                <figure class="highlight angelscript">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="keyword">from</span> scrapy <span class="keyword">import</span> Item, Field</span><br><span class="line"></span><br><span class="line"><span class="keyword">class</span> <span class="symbol">UserItem</span>(<span class="symbol">Item</span>):</span><br><span class="line">    <span class="symbol">collection</span> = '<span class="symbol">users</span>'</span><br><span class="line">    <span class="symbol">id</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">name</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">avatar</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">cover</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">gender</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">description</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">fans_count</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">follows_count</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">weibos_count</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">verified</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">verified_reason</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">verified_type</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">follows</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">fans</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">crawled_at</span> = <span class="symbol">Field</span>()</span><br><span class="line"></span><br><span class="line"><span class="symbol">class</span> <span class="symbol">UserRelationItem</span>(<span class="symbol">Item</span>):</span><br><span class="line">    <span class="symbol">collection</span> = '<span class="symbol">users</span>'</span><br><span class="line">    <span class="symbol">id</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">follows</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">fans</span> = <span class="symbol">Field</span>()</span><br><span class="line"></span><br><span class="line"><span class="symbol">class</span> <span class="symbol">WeiboItem</span>(<span class="symbol">Item</span>):</span><br><span class="line">    <span class="symbol">collection</span> = '<span class="symbol">weibos</span>'</span><br><span class="line">    <span class="symbol">id</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">attitudes_count</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">comments_count</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">reposts_count</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">picture</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">pictures</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">source</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">text</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">raw_text</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">thumbnail</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">user</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">created_at</span> = <span class="symbol">Field</span>()</span><br><span class="line">    <span class="symbol">crawled_at</span> = <span class="symbol">Field</span>()</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>这里定义了 collection 字段，指明保存的 Collection 的名称。用户的关注和粉丝列表直接定义为一个单独的 UserRelationItem，其中 id 就是用户的 ID，follows 就是用户关注列表，fans 是粉丝列表，但这并不意味着我们会将关注和粉丝列表存到一个单独的 Collection 里。后面我们会用 Pipeline 对各个 Item 进行处理、合并存储到用户的 Collection 里，因此 Item 和 Collection 并不一定是完全对应的。</p>
                <h3 id="7-提取数据"><a href="#7-提取数据" class="headerlink" title="7. 提取数据"></a>7. 提取数据</h3>
                <p>我们开始解析用户的基本信息，实现 parse_user() 方法，如下所示：</p>
                <figure class="highlight routeros">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">def parse_user(self, response):</span><br><span class="line">    <span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">    解析用户信息</span></span><br><span class="line"><span class="string">    :param response: Response 对象</span></span><br><span class="line"><span class="string">    "</span><span class="string">""</span></span><br><span class="line">    result = json.loads(response.text)</span><br><span class="line">    <span class="keyword">if</span> result.<span class="builtin-name">get</span>(<span class="string">'userInfo'</span>):</span><br><span class="line">        user_info = result.<span class="builtin-name">get</span>(<span class="string">'userInfo'</span>)</span><br><span class="line">        user_item = UserItem()</span><br><span class="line">        field_map = &#123;</span><br><span class="line">            <span class="string">'id'</span>: <span class="string">'id'</span>, <span class="string">'name'</span>: <span class="string">'screen_name'</span>, <span class="string">'avatar'</span>: <span class="string">'profile_image_url'</span>, <span class="string">'cover'</span>: <span class="string">'cover_image_phone'</span>,</span><br><span class="line">            <span class="string">'gender'</span>: <span class="string">'gender'</span>, <span class="string">'description'</span>: <span class="string">'description'</span>, <span class="string">'fans_count'</span>: <span class="string">'followers_count'</span>,</span><br><span class="line">            <span class="string">'follows_count'</span>: <span class="string">'follow_count'</span>, <span class="string">'weibos_count'</span>: <span class="string">'statuses_count'</span>, <span class="string">'verified'</span>: <span class="string">'verified'</span>,</span><br><span class="line">            <span class="string">'verified_reason'</span>: <span class="string">'verified_reason'</span>, <span class="string">'verified_type'</span>: <span class="string">'verified_type'</span></span><br><span class="line">        &#125;</span><br><span class="line">        <span class="keyword">for</span> field, attr <span class="keyword">in</span> field_map.items():</span><br><span class="line">            user_item[field] = user_info.<span class="builtin-name">get</span>(attr)</span><br><span class="line">        yield user_item</span><br><span class="line">        # 关注</span><br><span class="line">        uid = user_info.<span class="builtin-name">get</span>(<span class="string">'id'</span>)</span><br><span class="line">        yield Request(self.follow_url.format(<span class="attribute">uid</span>=uid, <span class="attribute">page</span>=1), <span class="attribute">callback</span>=self.parse_follows,</span><br><span class="line">                      meta=&#123;<span class="string">'page'</span>: 1, <span class="string">'uid'</span>: uid&#125;)</span><br><span class="line">        # 粉丝</span><br><span class="line">        yield Request(self.fan_url.format(<span class="attribute">uid</span>=uid, <span class="attribute">page</span>=1), <span class="attribute">callback</span>=self.parse_fans,</span><br><span class="line">                      meta=&#123;<span class="string">'page'</span>: 1, <span class="string">'uid'</span>: uid&#125;)</span><br><span class="line">        # 微博</span><br><span class="line">        yield Request(self.weibo_url.format(<span class="attribute">uid</span>=uid, <span class="attribute">page</span>=1), <span class="attribute">callback</span>=self.parse_weibos,</span><br><span class="line">                      meta=&#123;<span class="string">'page'</span>: 1, <span class="string">'uid'</span>: uid&#125;)</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>在这里我们一共完成了两个操作。</p>
                <ul>
                  <li>解析 JSON 提取用户信息并生成 UserItem 返回。我们并没有采用常规的逐个赋值的方法，而是定义了一个字段映射关系。我们定义的字段名称可能和 JSON 中用户的字段名称不同，所以在这里定义成一个字典，然后遍历字典的每个字段实现逐个字段的赋值。</li>
                  <li>构造用户的关注、粉丝、微博的第一页的链接，并生成 Request，这里需要的参数只有用户的 ID。另外，初始分页页码直接设置为 1 即可。</li>
                </ul>
                <p>接下来，我们还需要保存用户的关注和粉丝列表。以关注列表为例，其解析方法为 parse_follows()，实现如下所示：</p>
                <figure class="highlight routeros">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">def parse_follows(self, response):</span><br><span class="line">    <span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">    解析用户关注</span></span><br><span class="line"><span class="string">    :param response: Response 对象</span></span><br><span class="line"><span class="string">    "</span><span class="string">""</span></span><br><span class="line">    result = json.loads(response.text)</span><br><span class="line">    <span class="keyword">if</span> result.<span class="builtin-name">get</span>(<span class="string">'ok'</span>) <span class="keyword">and</span> result.<span class="builtin-name">get</span>(<span class="string">'cards'</span>) <span class="keyword">and</span> len(result.<span class="builtin-name">get</span>(<span class="string">'cards'</span>)) <span class="keyword">and</span> result.<span class="builtin-name">get</span>(<span class="string">'cards'</span>)[-1].<span class="builtin-name">get</span>(<span class="string">'card_group'</span>):</span><br><span class="line">        # 解析用户</span><br><span class="line">        follows = result.<span class="builtin-name">get</span>(<span class="string">'cards'</span>)[-1].<span class="builtin-name">get</span>(<span class="string">'card_group'</span>)</span><br><span class="line">        <span class="keyword">for</span> follow <span class="keyword">in</span> follows:</span><br><span class="line">            <span class="keyword">if</span> follow.<span class="builtin-name">get</span>(<span class="string">'user'</span>):</span><br><span class="line">                uid = follow.<span class="builtin-name">get</span>(<span class="string">'user'</span>).<span class="builtin-name">get</span>(<span class="string">'id'</span>)</span><br><span class="line">                yield Request(self.user_url.format(<span class="attribute">uid</span>=uid), <span class="attribute">callback</span>=self.parse_user)</span><br><span class="line">        # 关注列表</span><br><span class="line">        uid = response.meta.<span class="builtin-name">get</span>(<span class="string">'uid'</span>)</span><br><span class="line">        user_relation_item = UserRelationItem()</span><br><span class="line">        follows = [&#123;<span class="string">'id'</span>: follow.<span class="builtin-name">get</span>(<span class="string">'user'</span>).<span class="builtin-name">get</span>(<span class="string">'id'</span>), <span class="string">'name'</span>: follow.<span class="builtin-name">get</span>(<span class="string">'user'</span>).<span class="builtin-name">get</span>(<span class="string">'screen_name'</span>)&#125; <span class="keyword">for</span> follow <span class="keyword">in</span></span><br><span class="line">                   follows]</span><br><span class="line">        user_relation_item[<span class="string">'id'</span>] = uid</span><br><span class="line">        user_relation_item[<span class="string">'follows'</span>] = follows</span><br><span class="line">        user_relation_item[<span class="string">'fans'</span>] = []</span><br><span class="line">        yield user_relation_item</span><br><span class="line">        # 下一页关注</span><br><span class="line">       <span class="built_in"> page </span>= response.meta.<span class="builtin-name">get</span>(<span class="string">'page'</span>) + 1</span><br><span class="line">        yield Request(self.follow_url.format(<span class="attribute">uid</span>=uid, <span class="attribute">page</span>=page),</span><br><span class="line">                      <span class="attribute">callback</span>=self.parse_follows, meta=&#123;<span class="string">'page'</span>: page, <span class="string">'uid'</span>: uid&#125;)</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>那么在这个方法里面我们做了如下三件事。</p>
                <ul>
                  <li>解析关注列表中的每个用户信息并发起新的解析请求。我们首先解析关注列表的信息，得到用户的 ID，然后再利用 user_url 构造访问用户详情的 Request，回调就是刚才所定义的 parse_user() 方法。</li>
                  <li>提取用户关注列表内的关键信息并生成 UserRelationItem。id 字段直接设置成用户的 ID，JSON 返回数据中的用户信息有很多冗余字段。在这里我们只提取了关注用户的 ID 和用户名，然后把它们赋值给 follows 字段，fans 字段设置成空列表。这样我们就建立了一个存有用户 ID 和用户部分关注列表的 UserRelationItem，之后合并且保存具有同一个 ID 的 UserRelationItem 的关注和粉丝列表。</li>
                  <li>提取下一页关注。只需要将此请求的分页页码加 1 即可。分页页码通过 Request 的 meta 属性进行传递，Response 的 meta 来接收。这样我们构造并返回下一页的关注列表的 Request。</li>
                </ul>
                <p>抓取粉丝列表的原理和抓取关注列表原理相同，在此不再赘述。 接下来我们还差一个方法的实现，即 parse_weibos()，它用来抓取用户的微博信息，实现如下所示：</p>
                <figure class="highlight routeros">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">def parse_weibos(self, response):</span><br><span class="line">    <span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">    解析微博列表</span></span><br><span class="line"><span class="string">    :param response: Response 对象</span></span><br><span class="line"><span class="string">    "</span><span class="string">""</span></span><br><span class="line">    result = json.loads(response.text)</span><br><span class="line">    <span class="keyword">if</span> result.<span class="builtin-name">get</span>(<span class="string">'ok'</span>) <span class="keyword">and</span> result.<span class="builtin-name">get</span>(<span class="string">'cards'</span>):</span><br><span class="line">        weibos = result.<span class="builtin-name">get</span>(<span class="string">'cards'</span>)</span><br><span class="line">        <span class="keyword">for</span> weibo <span class="keyword">in</span> weibos:</span><br><span class="line">            mblog = weibo.<span class="builtin-name">get</span>(<span class="string">'mblog'</span>)</span><br><span class="line">            <span class="keyword">if</span> mblog:</span><br><span class="line">                weibo_item = WeiboItem()</span><br><span class="line">                field_map = &#123;</span><br><span class="line">                    <span class="string">'id'</span>: <span class="string">'id'</span>, <span class="string">'attitudes_count'</span>: <span class="string">'attitudes_count'</span>, <span class="string">'comments_count'</span>: <span class="string">'comments_count'</span>, <span class="string">'created_at'</span>: <span class="string">'created_at'</span>,</span><br><span class="line">                    <span class="string">'reposts_count'</span>: <span class="string">'reposts_count'</span>, <span class="string">'picture'</span>: <span class="string">'original_pic'</span>, <span class="string">'pictures'</span>: <span class="string">'pics'</span>,</span><br><span class="line">                    <span class="string">'source'</span>: <span class="string">'source'</span>, <span class="string">'text'</span>: <span class="string">'text'</span>, <span class="string">'raw_text'</span>: <span class="string">'raw_text'</span>, <span class="string">'thumbnail'</span>: <span class="string">'thumbnail_pic'</span></span><br><span class="line">                &#125;</span><br><span class="line">                <span class="keyword">for</span> field, attr <span class="keyword">in</span> field_map.items():</span><br><span class="line">                    weibo_item[field] = mblog.<span class="builtin-name">get</span>(attr)</span><br><span class="line">                weibo_item[<span class="string">'user'</span>] = response.meta.<span class="builtin-name">get</span>(<span class="string">'uid'</span>)</span><br><span class="line">                yield weibo_item</span><br><span class="line">        # 下一页微博</span><br><span class="line">        uid = response.meta.<span class="builtin-name">get</span>(<span class="string">'uid'</span>)</span><br><span class="line">       <span class="built_in"> page </span>= response.meta.<span class="builtin-name">get</span>(<span class="string">'page'</span>) + 1</span><br><span class="line">        yield Request(self.weibo_url.format(<span class="attribute">uid</span>=uid, <span class="attribute">page</span>=page), <span class="attribute">callback</span>=self.parse_weibos,</span><br><span class="line">                      meta=&#123;<span class="string">'uid'</span>: uid, <span class="string">'page'</span>: page&#125;)</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>这里 parse_weibos() 方法完成了两件事。</p>
                <ul>
                  <li>提取用户的微博信息，并生成 WeiboItem。这里同样建立了一个字段映射表，实现批量字段赋值。</li>
                  <li>提取下一页的微博列表。这里同样需要传入用户 ID 和分页页码。</li>
                </ul>
                <p>到目前为止，微博的 Spider 已经完成。后面还需要对数据进行数据清洗存储，以及对接代理池、Cookies 池来防止反爬虫。</p>
                <h3 id="8-数据清洗"><a href="#8-数据清洗" class="headerlink" title="8. 数据清洗"></a>8. 数据清洗</h3>
                <p>有些微博的时间可能不是标准的时间，比如它可能显示为刚刚、几分钟前、几小时前、昨天等。这里我们需要统一转化这些时间，实现一个 parse_time() 方法，如下所示：</p>
                <figure class="highlight pgsql">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">def parse_time(self, <span class="type">date</span>):</span><br><span class="line">    <span class="keyword">if</span> re.match(<span class="string">' 刚刚 '</span>, <span class="type">date</span>):</span><br><span class="line">        <span class="type">date</span> = <span class="type">time</span>.strftime(<span class="string">'% Y-% m-% d % H:% M'</span>, <span class="type">time</span>.<span class="built_in">localtime</span>(<span class="type">time</span>.time()))</span><br><span class="line">    <span class="keyword">if</span> re.match(<span class="string">'d + 分钟前 '</span>, <span class="type">date</span>):</span><br><span class="line">        minute = re.match(<span class="string">'(d+)'</span>, <span class="type">date</span>).<span class="keyword">group</span>(<span class="number">1</span>)</span><br><span class="line">        <span class="type">date</span> = <span class="type">time</span>.strftime(<span class="string">'% Y-% m-% d % H:% M'</span>, <span class="type">time</span>.<span class="built_in">localtime</span>(<span class="type">time</span>.time() - <span class="type">float</span>(minute) * <span class="number">60</span>))</span><br><span class="line">    <span class="keyword">if</span> re.match(<span class="string">'d + 小时前 '</span>, <span class="type">date</span>):</span><br><span class="line">        hour = re.match(<span class="string">'(d+)'</span>, <span class="type">date</span>).<span class="keyword">group</span>(<span class="number">1</span>)</span><br><span class="line">        <span class="type">date</span> = <span class="type">time</span>.strftime(<span class="string">'% Y-% m-% d % H:% M'</span>, <span class="type">time</span>.<span class="built_in">localtime</span>(<span class="type">time</span>.time() - <span class="type">float</span>(hour) * <span class="number">60</span> * <span class="number">60</span>))</span><br><span class="line">    <span class="keyword">if</span> re.match(<span class="string">' 昨天.*'</span>, <span class="type">date</span>):</span><br><span class="line">        <span class="type">date</span> = re.match(<span class="string">' 昨天 (.*)'</span>, <span class="type">date</span>).<span class="keyword">group</span>(<span class="number">1</span>).strip()</span><br><span class="line">        <span class="type">date</span> = <span class="type">time</span>.strftime(<span class="string">'% Y-% m-% d'</span>, <span class="type">time</span>.<span class="built_in">localtime</span>() - <span class="number">24</span> * <span class="number">60</span> * <span class="number">60</span>) + <span class="string">' '</span> + <span class="type">date</span></span><br><span class="line">    <span class="keyword">if</span> re.match(<span class="string">'d&#123;2&#125;-d&#123;2&#125;'</span>, <span class="type">date</span>):</span><br><span class="line">        <span class="type">date</span> = <span class="type">time</span>.strftime(<span class="string">'% Y-'</span>, <span class="type">time</span>.<span class="built_in">localtime</span>()) + <span class="type">date</span> + <span class="string">' 00:00'</span></span><br><span class="line">    <span class="keyword">return</span> <span class="type">date</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>我们用正则来提取一些关键数字，用 time 库来实现标准时间的转换。 以 X 分钟前的处理为例，爬取的时间会赋值为 created_at 字段。我们首先用正则匹配这个时间，表达式写作 d + 分钟前，如果提取到的时间符合这个表达式，那么就提取出其中的数字，这样就可以获取分钟数了。接下来使用 time 模块的 strftime() 方法，第一个参数传入要转换的时间格式，第二个参数就是时间戳。这里我们用当前的时间戳减去此分钟数乘以 60 就是当时的时间戳，这样我们就可以得到格式化后的正确时间了。 然后 Pipeline 可以实现如下处理：</p>
                <figure class="highlight livecodeserver">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">class WeiboPipeline():</span><br><span class="line">    def process_item(self, <span class="keyword">item</span>, spider):</span><br><span class="line">        <span class="keyword">if</span> isinstance(<span class="keyword">item</span>, WeiboItem):</span><br><span class="line">            <span class="keyword">if</span> <span class="keyword">item</span>.<span class="built_in">get</span>(<span class="string">'created_at'</span>):</span><br><span class="line">                <span class="keyword">item</span>[<span class="string">'created_at'</span>] = <span class="keyword">item</span>[<span class="string">'created_at'</span>].strip()</span><br><span class="line">                <span class="keyword">item</span>[<span class="string">'created_at'</span>] = self.parse_time(<span class="keyword">item</span>.<span class="built_in">get</span>(<span class="string">'created_at'</span>))</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>我们在 Spider 里没有对 crawled_at 字段赋值，它代表爬取时间，我们可以统一将其赋值为当前时间，实现如下所示：</p>
                <figure class="highlight haskell">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="type">TimePipeline</span>():</span></span><br><span class="line"><span class="class">    def process_item(<span class="title">self</span>, <span class="title">item</span>, <span class="title">spider</span>):</span></span><br><span class="line"><span class="class">        if isinstance(<span class="title">item</span>, <span class="type">UserItem</span>) or isinstance(<span class="title">item</span>, <span class="type">WeiboItem</span>):</span></span><br><span class="line"><span class="class">            now = time.strftime('% <span class="type">Y</span>-% <span class="title">m</span>-% <span class="title">d</span> % <span class="type">H</span>:% <span class="type">M</span>', <span class="title">time</span>.<span class="title">localtime</span>())</span></span><br><span class="line"><span class="class">            item['crawled_at'] = now</span></span><br><span class="line"><span class="class">        return item</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>这里我们判断了 item 如果是 UserItem 或 WeiboItem 类型，那么就给它的 crawled_at 字段赋值为当前时间。 通过上面的两个 Pipeline，我们便完成了数据清洗工作，这里主要是时间的转换。</p>
                <h3 id="9-数据存储"><a href="#9-数据存储" class="headerlink" title="9. 数据存储"></a>9. 数据存储</h3>
                <p>数据清洗完毕之后，我们就要将数据保存到 MongoDB 数据库。我们在这里实现 MongoPipeline 类，如下所示：</p>
                <figure class="highlight ruby">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">import pymongo</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">MongoPipeline</span>(<span class="title">object</span>):</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>, mongo_uri, mongo_db)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.mongo_uri = mongo_uri</span><br><span class="line">        <span class="keyword">self</span>.mongo_db = mongo_db</span><br><span class="line"></span><br><span class="line">    @classmethod</span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span><span class="params">(cls, crawler)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">return</span> cls(mongo_uri=crawler.settings.get(<span class="string">'MONGO_URI'</span>), mongo_db=crawler.settings.get(<span class="string">'MONGO_DATABASE'</span>)</span><br><span class="line">        )</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">open_spider</span><span class="params">(<span class="keyword">self</span>, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.client = pymongo.MongoClient(<span class="keyword">self</span>.mongo_uri)</span><br><span class="line">        <span class="keyword">self</span>.db = <span class="keyword">self</span>.client[<span class="keyword">self</span>.mongo_db]</span><br><span class="line">        <span class="keyword">self</span>.db[UserItem.collection].create_index([(<span class="string">'id'</span>, pymongo.ASCENDING)])</span><br><span class="line">        <span class="keyword">self</span>.db[WeiboItem.collection].create_index([(<span class="string">'id'</span>, pymongo.ASCENDING)])</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">close_spider</span><span class="params">(<span class="keyword">self</span>, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">self</span>.client.close()</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_item</span><span class="params">(<span class="keyword">self</span>, item, spider)</span></span><span class="symbol">:</span></span><br><span class="line">        <span class="keyword">if</span> isinstance(item, UserItem) <span class="keyword">or</span> isinstance(item, WeiboItem)<span class="symbol">:</span></span><br><span class="line">            <span class="keyword">self</span>.db[item.collection].update(&#123;<span class="string">'id'</span>: item.get(<span class="string">'id'</span>)&#125;, &#123;<span class="string">'$set'</span>: item&#125;, True)</span><br><span class="line">        <span class="keyword">if</span> isinstance(item, UserRelationItem)<span class="symbol">:</span></span><br><span class="line">            <span class="keyword">self</span>.db[item.collection].update(&#123;<span class="string">'id'</span>: item.get(<span class="string">'id'</span>)&#125;,</span><br><span class="line">                &#123;<span class="string">'$addToSet'</span><span class="symbol">:</span></span><br><span class="line">                    &#123;<span class="string">'follows'</span>: &#123;<span class="string">'$each'</span>: item[<span class="string">'follows'</span>]&#125;,</span><br><span class="line">                        <span class="string">'fans'</span>: &#123;<span class="string">'$each'</span>: item[<span class="string">'fans'</span>]&#125;</span><br><span class="line">                    &#125;</span><br><span class="line">                &#125;, True)</span><br><span class="line">        <span class="keyword">return</span> item</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>当前的 MongoPipeline 和前面我们所写的有所不同，主要有以下几点。</p>
                <ul>
                  <li>在 open_spider() 方法里面添加了 Collection 的索引，在这里为两个 Item 都做了索引，索引的字段是 id，由于我们这次是大规模爬取，同时在爬取过程中涉及到数据的更新问题，所以我们为每个 Collection 建立了索引，建立了索引之后可以大大提高检索效率。</li>
                  <li>在 process_item() 方法里存储使用的是 update() 方法，第一个参数是查询条件，第二个参数是爬取的 Item，这里我们使用了 $set 操作符，这样我们如果爬取到了重复的数据即可对数据进行更新，同时不会删除已存在的字段，如果这里不加 $set 操作符，那么会直接进行 item 替换，这样可能会导致已存在的字段如关注和粉丝列表清空，所以这里必须要加上 $set 操作符。第三个参数我们设置为了 True，这个参数起到的作用是如果数据不存在，则插入数据。这样我们就可以做到数据存在即更新、数据不存在即插入，这样就达到了去重的效果。</li>
                  <li>对于用户的关注和粉丝列表，我们在这里使用了一个新的操作符，叫做 $addToSet，这个操作符可以向列表类型的字段插入数据同时去重，接下来它的值就是需要操作的字段名称，我们在这里又利用了 $each 操作符对需要插入的列表数据进行了遍历，这样就可以逐条插入用户的关注或粉丝数据到指定的字段了，关于该操作更多的解释可以参考 MongoDB 的官方文档，链接为：<a href="https://docs.mongodb.com/manual/reference/operator/update/addToSet/" target="_blank" rel="noopener">https://docs.mongodb.com/manual/reference/operator/update/addToSet/</a>。</li>
                </ul>
                <h3 id="10-Cookies-池对接"><a href="#10-Cookies-池对接" class="headerlink" title="10. Cookies 池对接"></a>10. Cookies 池对接</h3>
                <p>新浪微博的反爬能力非常强，我们需要做一些防范反爬虫的措施才可以顺利完成数据爬取。 如果没有登录而直接请求微博的 API 接口，这非常容易导致 403 状态码。这个情况我们在 10.2 节也提过。所以在这里我们实现一个 Middleware，为每个 Request 添加随机的 Cookies。 我们先开启 Cookies 池，使 API 模块正常运行。例如在本地运行 5000 端口，访问：<a href="http://localhost:5000/weibo/random" target="_blank" rel="noopener">http://localhost:5000/weibo/random</a> 即可获取随机的 Cookies，当然也可以将 Cookies 池部署到远程的服务器，这样只需要更改一下访问的链接就好了。 那么在这里我们将 Cookies 池在本地启动起来，再实现一个 Middleware 如下：</p>
                <figure class="highlight ruby">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">CookiesMiddleware</span>():</span></span><br><span class="line">   <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(<span class="keyword">self</span>, cookies_url)</span></span><span class="symbol">:</span></span><br><span class="line">       <span class="keyword">self</span>.logger = logging.getLogger(__name_<span class="number">_</span>)</span><br><span class="line">       <span class="keyword">self</span>.cookies_url = cookies_url</span><br><span class="line"></span><br><span class="line">   <span class="function"><span class="keyword">def</span> <span class="title">get_random_cookies</span><span class="params">(<span class="keyword">self</span>)</span></span><span class="symbol">:</span></span><br><span class="line">       <span class="symbol">try:</span></span><br><span class="line">           response = requests.get(<span class="keyword">self</span>.cookies_url)</span><br><span class="line">           <span class="keyword">if</span> response.status_code == <span class="number">200</span><span class="symbol">:</span></span><br><span class="line">               cookies = json.loads(response.text)</span><br><span class="line">               <span class="keyword">return</span> cookies</span><br><span class="line">       except requests.<span class="symbol">ConnectionError:</span></span><br><span class="line">           <span class="keyword">return</span> False</span><br><span class="line"></span><br><span class="line">   <span class="function"><span class="keyword">def</span> <span class="title">process_request</span><span class="params">(<span class="keyword">self</span>, request, spider)</span></span><span class="symbol">:</span></span><br><span class="line">       <span class="keyword">self</span>.logger.debug(<span class="string">' 正在获取 Cookies'</span>)</span><br><span class="line">       cookies = <span class="keyword">self</span>.get_random_cookies()</span><br><span class="line">       <span class="keyword">if</span> <span class="symbol">cookies:</span></span><br><span class="line">           request.cookies = cookies</span><br><span class="line">           <span class="keyword">self</span>.logger.debug(<span class="string">' 使用 Cookies '</span> + json.dumps(cookies))</span><br><span class="line"></span><br><span class="line">   @classmethod</span><br><span class="line">   <span class="function"><span class="keyword">def</span> <span class="title">from_crawler</span><span class="params">(cls, crawler)</span></span><span class="symbol">:</span></span><br><span class="line">       settings = crawler.settings</span><br><span class="line">       <span class="keyword">return</span> cls(cookies_url=settings.get(<span class="string">'COOKIES_URL'</span>)</span><br><span class="line">       )</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>我们首先利用 from_crawler() 方法获取了 COOKIES_URL 变量，它定义在 settings.py 里，这就是刚才我们所说的接口。接下来实现 get_random_cookies() 方法，这个方法主要就是请求此 Cookies 池接口并获取接口返回的随机 Cookies。如果成功获取，则返回 Cookies；否则返回 False。 接下来，在 process_request() 方法里，我们给 request 对象的 cookies 属性赋值，其值就是获取的随机 Cookies，这样我们就成功地为每一次请求赋值 Cookies 了。 如果启用了该 Middleware，每个请求都会被赋值随机的 Cookies。这样我们就可以模拟登录之后的请求，403 状态码基本就不会出现。</p>
                <h3 id="11-代理池对接"><a href="#11-代理池对接" class="headerlink" title="11. 代理池对接"></a>11. 代理池对接</h3>
                <p>微博还有一个反爬措施就是，检测到同一 IP 请求量过大时就会出现 414 状态码。如果遇到这样的情况可以切换代理。例如，在本地 5555 端口运行，获取随机可用代理的地址为：<a href="http://localhost:5555/random" target="_blank" rel="noopener">http://localhost:5555/random</a>，访问这个接口即可获取一个随机可用代理。接下来我们再实现一个 Middleware，代码如下所示：</p>
                <figure class="highlight routeros">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line">class ProxyMiddleware():</span><br><span class="line">    def __init__(self, proxy_url):</span><br><span class="line">        self.logger = logging.getLogger(__name__)</span><br><span class="line">        self.proxy_url = proxy_url</span><br><span class="line"></span><br><span class="line">    def get_random_proxy(self):</span><br><span class="line">        try:</span><br><span class="line">            response = requests.<span class="builtin-name">get</span>(self.proxy_url)</span><br><span class="line">            <span class="keyword">if</span> response.status_code == 200:</span><br><span class="line">               <span class="built_in"> proxy </span>= response.text</span><br><span class="line">                return proxy</span><br><span class="line">        except requests.ConnectionError:</span><br><span class="line">            return <span class="literal">False</span></span><br><span class="line"></span><br><span class="line">    def process_request(self, request, spider):</span><br><span class="line">        <span class="keyword">if</span> request.meta.<span class="builtin-name">get</span>(<span class="string">'retry_times'</span>):</span><br><span class="line">           <span class="built_in"> proxy </span>= self.get_random_proxy()</span><br><span class="line">            <span class="keyword">if</span> proxy:</span><br><span class="line">                uri = <span class="string">'https://&#123;proxy&#125;'</span>.format(<span class="attribute">proxy</span>=proxy)</span><br><span class="line">                self.logger.<span class="builtin-name">debug</span>(<span class="string">' 使用代理 '</span> + proxy)</span><br><span class="line">                request.meta[<span class="string">'proxy'</span>] = uri</span><br><span class="line"></span><br><span class="line">    @classmethod</span><br><span class="line">    def from_crawler(cls, crawler):</span><br><span class="line">       <span class="built_in"> settings </span>= crawler.settings</span><br><span class="line">        return cls(<span class="attribute">proxy_url</span>=settings.get('PROXY_URL')</span><br><span class="line">        )</span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>同样的原理，我们实现了一个 get_random_proxy() 方法用于请求代理池的接口获取随机代理。如果获取成功，则返回改代理，否则返回 False。在 process_request() 方法中，我们给 request 对象的 meta 属性赋值一个 proxy 字段，该字段的值就是代理。 另外，赋值代理的判断条件是当前 retry_times 不为空，也就是说第一次请求失败之后才启用代理，因为使用代理后访问速度会慢一些。所以我们在这里设置了只有重试的时候才启用代理，否则直接请求。这样就可以保证在没有被封禁的情况下直接爬取，保证了爬取速度。</p>
                <h3 id="12-启用-Middleware"><a href="#12-启用-Middleware" class="headerlink" title="12. 启用 Middleware"></a>12. 启用 Middleware</h3>
                <p>接下来，我们在配置文件中启用这两个 Middleware，修改 settings.py 如下所示：</p>
                <figure class="highlight yaml">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="string">DOWNLOADER_MIDDLEWARES</span> <span class="string">=</span> <span class="string">&#123;</span></span><br><span class="line">    <span class="attr">'weibo.middlewares.CookiesMiddleware':</span> <span class="number">554</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'weibo.middlewares.ProxyMiddleware':</span> <span class="number">555</span><span class="string">,</span></span><br><span class="line"><span class="string">&#125;</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>注意这里的优先级设置，前文提到了 Scrapy 的默认 Downloader Middleware 的设置如下：</p>
                <figure class="highlight yaml">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="string">&#123;</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware':</span> <span class="number">100</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware':</span> <span class="number">300</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware':</span> <span class="number">350</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware':</span> <span class="number">400</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware':</span> <span class="number">500</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.retry.RetryMiddleware':</span> <span class="number">550</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware':</span> <span class="number">560</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware':</span> <span class="number">580</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware':</span> <span class="number">590</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.redirect.RedirectMiddleware':</span> <span class="number">600</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.cookies.CookiesMiddleware':</span> <span class="number">700</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware':</span> <span class="number">750</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.stats.DownloaderStats':</span> <span class="number">850</span><span class="string">,</span></span><br><span class="line">    <span class="attr">'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware':</span> <span class="number">900</span><span class="string">,</span></span><br><span class="line"><span class="string">&#125;</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>要使得我们自定义的 CookiesMiddleware 生效，它在内置的 CookiesMiddleware 之前调用。内置的 CookiesMiddleware 的优先级为 700，所以这里我们设置一个比 700 小的数字即可。 要使得我们自定义的 ProxyMiddleware 生效，它在内置的 HttpProxyMiddleware 之前调用。内置的 HttpProxyMiddleware 的优先级为 750，所以这里我们设置一个比 750 小的数字即可。</p>
                <h3 id="13-运行"><a href="#13-运行" class="headerlink" title="13. 运行"></a>13. 运行</h3>
                <p>到此为止，整个微博爬虫就实现完毕了，我们运行如下命令启动一下爬虫：</p>
                <figure class="highlight ebnf">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="attribute">scrapy crawl weibocn</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>类似的输出结果如下：</p>
                <figure class="highlight yaml">
                  <table>
                    <tr>
                      <td class="gutter">
                        <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                      </td>
                      <td class="code">
                        <pre><span class="line"><span class="number">2017</span><span class="number">-07</span><span class="number">-11</span> <span class="number">17</span><span class="string">:27:34</span> <span class="string">[urllib3.connectionpool]</span> <span class="attr">DEBUG:</span> <span class="string">http://localhost:5000</span> <span class="string">"GET /weibo/random HTTP/1.1"</span> <span class="number">200</span> <span class="number">339</span></span><br><span class="line"><span class="number">2017</span><span class="number">-07</span><span class="number">-11</span> <span class="number">17</span><span class="string">:27:34</span> <span class="string">[weibo.middlewares]</span> <span class="attr">DEBUG:</span> <span class="string">使用</span> <span class="string">Cookies</span> <span class="string">&#123;"SCF":</span> <span class="string">"AhzwTr_DxIGjgri_dt46_DoPzUqq-PSupu545JdozdHYJ7HyEb4pD3pe05VpbIpVyY1ciKRRWwUgojiO3jYwlBE."</span><span class="string">,</span> <span class="attr">"_T_WM":</span> <span class="string">"8fe0bc1dad068d09b888d8177f1c1218"</span><span class="string">,</span> <span class="attr">"SSOLoginState":</span> <span class="string">"1501496388"</span><span class="string">,</span> <span class="attr">"M_WEIBOCN_PARAMS":</span> <span class="string">"uicode%3D20000174"</span><span class="string">,</span> <span class="attr">"SUHB":</span> <span class="string">"0tKqV4asxqYl4J"</span><span class="string">,</span> <span class="attr">"SUB":</span> <span class="string">"_2A250e3QUDeRhGeBM6VYX8y7NwjiIHXVXhBxcrDV6PUJbkdBeLXjckW2fUT8MWloekO4FCWVlIYJGJdGLnA.."</span><span class="string">&#125;</span></span><br><span class="line"><span class="number">2017</span><span class="number">-07</span><span class="number">-11</span> <span class="number">17</span><span class="string">:27:34</span> <span class="string">[weibocn]</span> <span class="attr">DEBUG:</span> <span class="string">&lt;200</span> <span class="string">https://m.weibo.cn/api/container/getIndex?uid=1742566624&amp;type=uid&amp;value=1742566624&amp;containerid=1005051742566624&gt;</span></span><br><span class="line"><span class="number">2017</span><span class="number">-07</span><span class="number">-11</span> <span class="number">17</span><span class="string">:27:34</span> <span class="string">[scrapy.core.scraper]</span> <span class="attr">DEBUG:</span> <span class="string">Scraped</span> <span class="string">from</span> <span class="string">&lt;200</span> <span class="string">https://m.weibo.cn/api/container/getIndex?uid=1742566624&amp;type=uid&amp;value=1742566624&amp;containerid=1005051742566624&gt;</span></span><br><span class="line"><span class="string">&#123;'avatar':</span> <span class="string">'https://tva4.sinaimg.cn/crop.0.0.180.180.180/67dd74e0jw1e8qgp5bmzyj2050050aa8.jpg'</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'cover':</span> <span class="string">'https://tva3.sinaimg.cn/crop.0.0.640.640.640/6ce2240djw1e9oaqhwllzj20hs0hsdir.jpg'</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'crawled_at':</span> <span class="string">'2017-07-11 17:27'</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'description':</span> <span class="string">' 成长，就是一个不断觉得以前的自己是个傻逼的过程 '</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'fans_count':</span> <span class="number">19202906</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'follows_count':</span> <span class="number">1599</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'gender':</span> <span class="string">'m'</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'id':</span> <span class="number">1742566624</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'name':</span> <span class="string">' 思想聚焦 '</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'verified':</span> <span class="literal">True</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'verified_reason':</span> <span class="string">' 微博知名博主，校导网编辑 '</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'verified_type':</span> <span class="number">0</span><span class="string">,</span></span><br><span class="line"> <span class="attr">'weibos_count':</span> <span class="number">58393</span><span class="string">&#125;</span></span><br></pre>
                      </td>
                    </tr>
                  </table>
                </figure>
                <p>运行一段时间后，我们便可以到 MongoDB 数据库查看数据，爬取下来的数据如图 13-38 和图 13-39 所示。 <img src="https://qiniu.cuiqingcai.com/2019-11-27-034947.jpg" alt=""> 图 13-38 用户信息 <img src="https://qiniu.cuiqingcai.com/2019-11-27-035001.jpg" alt=""> 图 13-39 微博信息 针对用户信息，我们不仅爬取了其基本信息，还把关注和粉丝列表加到了 follows 和 fans 字段并做了去重操作。针对微博信息，我们成功进行了时间转换处理，同时还保存了微博的图片列表信息。</p>
                <h3 id="14-本节代码"><a href="#14-本节代码" class="headerlink" title="14. 本节代码"></a>14. 本节代码</h3>
                <p>本节代码地址：<a href="https://github.com/Python3WebSpider/Weibo" target="_blank" rel="noopener">https://github.com/Python3WebSpider/Weibo</a>。</p>
                <h3 id="15-结语"><a href="#15-结语" class="headerlink" title="15. 结语"></a>15. 结语</h3>
                <p>本节实现了新浪微博的用户及其粉丝关注列表和微博信息的爬取，还对接了 Cookies 池和代理池来处理反爬虫。不过现在是针对单机的爬取，后面我们会将此项目修改为分布式爬虫，以进一步提高抓取效率。</p>
              </div>
              <div class="reward-container">
                <div></div>
                <button onclick="var qr = document.getElementById('qr'); qr.style.display = (qr.style.display === 'none') ? 'block' : 'none';"> 打赏 </button>
                <div id="qr" style="display: none;">
                  <div style="display: inline-block;">
                    <img src="/images/wechatpay.jpg" alt="崔庆才 微信支付">
                    <p>微信支付</p>
                  </div>
                  <div style="display: inline-block;">
                    <img src="/images/alipay.jpg" alt="崔庆才 支付宝">
                    <p>支付宝</p>
                  </div>
                </div>
              </div>
              <footer class="post-footer">
                <div class="post-nav">
                  <div class="post-nav-item">
                    <a href="/8448.html" rel="prev" title="[Python3网络爬虫开发实战] 13.12–Scrapy 对接 Docker">
                      <i class="fa fa-chevron-left"></i> [Python3网络爬虫开发实战] 13.12–Scrapy 对接 Docker </a>
                  </div>
                  <div class="post-nav-item">
                    <a href="/8456.html" rel="next" title="[Python3网络爬虫开发实战] 14.1–分布式爬虫原理"> [Python3网络爬虫开发实战] 14.1–分布式爬虫原理 <i class="fa fa-chevron-right"></i>
                    </a>
                  </div>
                </div>
              </footer>
            </article>
          </div>
          <div class="comments" id="gitalk-container"></div>
          <script>
            window.addEventListener('tabs:register', () =>
            {
              let
              {
                activeClass
              } = CONFIG.comments;
              if (CONFIG.comments.storage)
              {
                activeClass = localStorage.getItem('comments_active') || activeClass;
              }
              if (activeClass)
              {
                let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
                if (activeTab)
                {
                  activeTab.click();
                }
              }
            });
            if (CONFIG.comments.storage)
            {
              window.addEventListener('tabs:click', event =>
              {
                if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
                let commentClass = event.target.classList[1];
                localStorage.setItem('comments_active', commentClass);
              });
            }

          </script>
        </div>
        <div class="toggle sidebar-toggle">
          <span class="toggle-line toggle-line-first"></span>
          <span class="toggle-line toggle-line-middle"></span>
          <span class="toggle-line toggle-line-last"></span>
        </div>
        <aside class="sidebar">
          <div class="sidebar-inner">
            <ul class="sidebar-nav motion-element">
              <li class="sidebar-nav-toc"> 文章目录 </li>
              <li class="sidebar-nav-overview"> 站点概览 </li>
            </ul>
            <!--noindex-->
            <div class="post-toc-wrap sidebar-panel">
              <div class="post-toc motion-element">
                <ol class="nav">
                  <li class="nav-item nav-level-1"><a class="nav-link" href="#13-13-Scrapy-爬取新浪微博"><span class="nav-number">1.</span> <span class="nav-text">13.13 Scrapy 爬取新浪微博</span></a>
                    <ol class="nav-child">
                      <li class="nav-item nav-level-3"><a class="nav-link" href="#1-本节目标"><span class="nav-number">1.0.1.</span> <span class="nav-text">1. 本节目标</span></a></li>
                      <li class="nav-item nav-level-3"><a class="nav-link" href="#2-准备工作"><span class="nav-number">1.0.2.</span> <span class="nav-text">2. 准备工作</span></a></li>
                      <li class="nav-item nav-level-3"><a class="nav-link" href="#3-爬取思路"><span class="nav-number">1.0.3.</span> <span class="nav-text">3. 爬取思路</span></a></li>
                      <li class="nav-item nav-level-3"><a class="nav-link" href="#4-爬取分析"><span class="nav-number">1.0.4.</span> <span class="nav-text">4. 爬取分析</span></a></li>
                      <li class="nav-item nav-level-3"><a class="nav-link" href="#5-新建项目"><span class="nav-number">1.0.5.</span> <span class="nav-text">5. 新建项目</span></a></li>
                      <li class="nav-item nav-level-3"><a class="nav-link" href="#6-创建-Item"><span class="nav-number">1.0.6.</span> <span class="nav-text">6. 创建 Item</span></a></li>
                      <li class="nav-item nav-level-3"><a class="nav-link" href="#7-提取数据"><span class="nav-number">1.0.7.</span> <span class="nav-text">7. 提取数据</span></a></li>
                      <li class="nav-item nav-level-3"><a class="nav-link" href="#8-数据清洗"><span class="nav-number">1.0.8.</span> <span class="nav-text">8. 数据清洗</span></a></li>
                      <li class="nav-item nav-level-3"><a class="nav-link" href="#9-数据存储"><span class="nav-number">1.0.9.</span> <span class="nav-text">9. 数据存储</span></a></li>
                      <li class="nav-item nav-level-3"><a class="nav-link" href="#10-Cookies-池对接"><span class="nav-number">1.0.10.</span> <span class="nav-text">10. Cookies 池对接</span></a></li>
                      <li class="nav-item nav-level-3"><a class="nav-link" href="#11-代理池对接"><span class="nav-number">1.0.11.</span> <span class="nav-text">11. 代理池对接</span></a></li>
                      <li class="nav-item nav-level-3"><a class="nav-link" href="#12-启用-Middleware"><span class="nav-number">1.0.12.</span> <span class="nav-text">12. 启用 Middleware</span></a></li>
                      <li class="nav-item nav-level-3"><a class="nav-link" href="#13-运行"><span class="nav-number">1.0.13.</span> <span class="nav-text">13. 运行</span></a></li>
                      <li class="nav-item nav-level-3"><a class="nav-link" href="#14-本节代码"><span class="nav-number">1.0.14.</span> <span class="nav-text">14. 本节代码</span></a></li>
                      <li class="nav-item nav-level-3"><a class="nav-link" href="#15-结语"><span class="nav-number">1.0.15.</span> <span class="nav-text">15. 结语</span></a></li>
                    </ol>
                  </li>
                </ol>
                </li>
                </ol>
              </div>
            </div>
            <!--/noindex-->
            <div class="site-overview-wrap sidebar-panel">
              <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
                <img class="site-author-image" itemprop="image" alt="崔庆才" src="/images/avatar.png">
                <p class="site-author-name" itemprop="name">崔庆才</p>
                <div class="site-description" itemprop="description">崔庆才的个人站点，记录生活的瞬间，分享学习的心得。</div>
              </div>
              <div class="site-state-wrap motion-element">
                <nav class="site-state">
                  <div class="site-state-item site-state-posts">
                    <a href="/archives/">
                      <span class="site-state-item-count">608</span>
                      <span class="site-state-item-name">日志</span>
                    </a>
                  </div>
                  <div class="site-state-item site-state-categories">
                    <a href="/categories/">
                      <span class="site-state-item-count">24</span>
                      <span class="site-state-item-name">分类</span></a>
                  </div>
                  <div class="site-state-item site-state-tags">
                    <a href="/tags/">
                      <span class="site-state-item-count">156</span>
                      <span class="site-state-item-name">标签</span></a>
                  </div>
                </nav>
              </div>
              <div class="links-of-author motion-element">
                <span class="links-of-author-item">
                  <a href="https://github.com/Germey" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;Germey" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>GitHub</a>
                </span>
                <span class="links-of-author-item">
                  <a href="mailto:cqc@cuiqingcai.com.com" title="邮件 → mailto:cqc@cuiqingcai.com.com" rel="noopener" target="_blank"><i class="fa fa-envelope fa-fw"></i>邮件</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://weibo.com/cuiqingcai" title="微博 → https:&#x2F;&#x2F;weibo.com&#x2F;cuiqingcai" rel="noopener" target="_blank"><i class="fab fa-weibo fa-fw"></i>微博</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://www.zhihu.com/people/Germey" title="知乎 → https:&#x2F;&#x2F;www.zhihu.com&#x2F;people&#x2F;Germey" rel="noopener" target="_blank"><i class="fa fa-magic fa-fw"></i>知乎</a>
                </span>
              </div>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/bco2a.png" style=" width: 100%;">
              </a>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/0ywun.png" style=" width: 100%;">
              </a>
            </div>
            <div class="sidebar-panel sidebar-panel-tags sidebar-panel-active">
              <h4 class="name"> 标签云 </h4>
              <div class="content">
                <a href="/tags/2048/" style="font-size: 10px;">2048</a> <a href="/tags/API/" style="font-size: 10px;">API</a> <a href="/tags/Bootstrap/" style="font-size: 11.25px;">Bootstrap</a> <a href="/tags/CDN/" style="font-size: 10px;">CDN</a> <a href="/tags/CQC/" style="font-size: 10px;">CQC</a> <a href="/tags/CSS/" style="font-size: 10px;">CSS</a> <a href="/tags/CSS-%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">CSS 反爬虫</a> <a href="/tags/CV/" style="font-size: 10px;">CV</a> <a href="/tags/Django/" style="font-size: 10px;">Django</a> <a href="/tags/Eclipse/" style="font-size: 11.25px;">Eclipse</a> <a href="/tags/FTP/" style="font-size: 10px;">FTP</a> <a href="/tags/Git/" style="font-size: 10px;">Git</a> <a href="/tags/GitHub/" style="font-size: 13.75px;">GitHub</a> <a href="/tags/HTML5/" style="font-size: 10px;">HTML5</a> <a href="/tags/Hexo/" style="font-size: 10px;">Hexo</a> <a href="/tags/IT/" style="font-size: 10px;">IT</a> <a href="/tags/JSP/" style="font-size: 10px;">JSP</a> <a href="/tags/JavaScript/" style="font-size: 10px;">JavaScript</a> <a href="/tags/K8s/" style="font-size: 10px;">K8s</a> <a href="/tags/LOGO/" style="font-size: 10px;">LOGO</a> <a href="/tags/Linux/" style="font-size: 10px;">Linux</a> <a href="/tags/MIUI/" style="font-size: 10px;">MIUI</a> <a href="/tags/MongoDB/" style="font-size: 10px;">MongoDB</a> <a href="/tags/Mysql/" style="font-size: 10px;">Mysql</a> <a href="/tags/NBA/" style="font-size: 10px;">NBA</a> <a href="/tags/PHP/" style="font-size: 11.25px;">PHP</a> <a href="/tags/PS/" style="font-size: 10px;">PS</a> <a href="/tags/Pathlib/" style="font-size: 10px;">Pathlib</a> <a href="/tags/PhantomJS/" style="font-size: 10px;">PhantomJS</a> <a href="/tags/Python/" style="font-size: 15px;">Python</a> <a href="/tags/Python3/" style="font-size: 12.5px;">Python3</a> <a href="/tags/Pythonic/" style="font-size: 10px;">Pythonic</a> <a href="/tags/QQ/" style="font-size: 10px;">QQ</a> <a href="/tags/Redis/" style="font-size: 10px;">Redis</a> <a href="/tags/SAE/" style="font-size: 10px;">SAE</a> <a href="/tags/SSH/" style="font-size: 10px;">SSH</a> <a href="/tags/SVG/" style="font-size: 10px;">SVG</a> <a href="/tags/Scrapy/" style="font-size: 10px;">Scrapy</a> <a href="/tags/Scrapy-redis/" style="font-size: 10px;">Scrapy-redis</a> <a href="/tags/Scrapy%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">Scrapy分布式</a> <a href="/tags/Selenium/" style="font-size: 10px;">Selenium</a> <a href="/tags/TKE/" style="font-size: 10px;">TKE</a> <a href="/tags/Ubuntu/" style="font-size: 11.25px;">Ubuntu</a> <a href="/tags/VS-Code/" style="font-size: 10px;">VS Code</a> <a href="/tags/Vs-Code/" style="font-size: 10px;">Vs Code</a> <a href="/tags/Vue/" style="font-size: 11.25px;">Vue</a> <a href="/tags/Webpack/" style="font-size: 10px;">Webpack</a> <a href="/tags/Windows/" style="font-size: 10px;">Windows</a> <a href="/tags/Winpcap/" style="font-size: 10px;">Winpcap</a> <a href="/tags/WordPress/" style="font-size: 13.75px;">WordPress</a> <a href="/tags/Youtube/" style="font-size: 11.25px;">Youtube</a> <a href="/tags/android/" style="font-size: 10px;">android</a> <a href="/tags/ansible/" style="font-size: 10px;">ansible</a> <a href="/tags/cocos2d-x/" style="font-size: 10px;">cocos2d-x</a> <a href="/tags/e6/" style="font-size: 10px;">e6</a> <a href="/tags/fitvids/" style="font-size: 10px;">fitvids</a> <a href="/tags/git/" style="font-size: 11.25px;">git</a> <a href="/tags/json/" style="font-size: 10px;">json</a> <a href="/tags/js%E9%80%86%E5%90%91/" style="font-size: 10px;">js逆向</a> <a href="/tags/kubernetes/" style="font-size: 10px;">kubernetes</a> <a href="/tags/log/" style="font-size: 10px;">log</a> <a href="/tags/logging/" style="font-size: 10px;">logging</a> <a href="/tags/matlab/" style="font-size: 11.25px;">matlab</a> <a href="/tags/python/" style="font-size: 20px;">python</a> <a href="/tags/pytube/" style="font-size: 11.25px;">pytube</a> <a href="/tags/pywin32/" style="font-size: 10px;">pywin32</a> <a href="/tags/style/" style="font-size: 10px;">style</a> <a href="/tags/tomcat/" style="font-size: 10px;">tomcat</a> <a href="/tags/ubuntu/" style="font-size: 10px;">ubuntu</a> <a href="/tags/uwsgi/" style="font-size: 10px;">uwsgi</a> <a href="/tags/vsftpd/" style="font-size: 10px;">vsftpd</a> <a href="/tags/wamp/" style="font-size: 10px;">wamp</a> <a href="/tags/wineQQ/" style="font-size: 10px;">wineQQ</a> <a href="/tags/%E4%B8%83%E7%89%9B/" style="font-size: 11.25px;">七牛</a> <a href="/tags/%E4%B8%8A%E6%B5%B7/" style="font-size: 10px;">上海</a> <a href="/tags/%E4%B8%AA%E4%BA%BA%E7%BD%91%E7%AB%99/" style="font-size: 10px;">个人网站</a> <a href="/tags/%E4%B8%BB%E9%A2%98/" style="font-size: 10px;">主题</a> <a href="/tags/%E4%BA%91%E4%BA%A7%E5%93%81/" style="font-size: 10px;">云产品</a> <a href="/tags/%E4%BA%91%E5%AD%98%E5%82%A8/" style="font-size: 10px;">云存储</a> <a href="/tags/%E4%BA%AC%E4%B8%9C%E4%BA%91/" style="font-size: 10px;">京东云</a> <a href="/tags/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD/" style="font-size: 12.5px;">人工智能</a> <a href="/tags/%E4%BB%A3%E7%90%86/" style="font-size: 10px;">代理</a> <a href="/tags/%E4%BB%A3%E7%A0%81/" style="font-size: 10px;">代码</a> <a href="/tags/%E4%BB%A3%E7%A0%81%E5%88%86%E4%BA%AB%E5%9B%BE/" style="font-size: 10px;">代码分享图</a> <a href="/tags/%E4%BC%98%E5%8C%96/" style="font-size: 10px;">优化</a> <a href="/tags/%E4%BD%8D%E8%BF%90%E7%AE%97/" style="font-size: 10px;">位运算</a> <a href="/tags/%E5%85%AC%E4%BC%97%E5%8F%B7/" style="font-size: 10px;">公众号</a> <a href="/tags/%E5%88%86%E4%BA%AB/" style="font-size: 10px;">分享</a> <a href="/tags/%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">分布式</a> <a href="/tags/%E5%88%9B%E4%B8%9A/" style="font-size: 10px;">创业</a> <a href="/tags/%E5%89%8D%E7%AB%AF/" style="font-size: 12.5px;">前端</a> <a href="/tags/%E5%8D%9A%E5%AE%A2/" style="font-size: 10px;">博客</a> <a href="/tags/%E5%8E%9F%E7%94%9FAPP/" style="font-size: 10px;">原生APP</a> <a href="/tags/%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 12.5px;">反爬虫</a> <a href="/tags/%E5%91%BD%E4%BB%A4/" style="font-size: 10px;">命令</a> <a href="/tags/%E5%93%8D%E5%BA%94%E5%BC%8F%E5%B8%83%E5%B1%80/" style="font-size: 10px;">响应式布局</a> <a href="/tags/%E5%9E%83%E5%9C%BE%E9%82%AE%E4%BB%B6/" style="font-size: 10px;">垃圾邮件</a> <a href="/tags/%E5%9F%9F%E5%90%8D%E7%BB%91%E5%AE%9A/" style="font-size: 10px;">域名绑定</a> <a href="/tags/%E5%A4%8D%E7%9B%98/" style="font-size: 10px;">复盘</a> <a href="/tags/%E5%A4%A7%E4%BC%97%E7%82%B9%E8%AF%84/" style="font-size: 10px;">大众点评</a> <a href="/tags/%E5%AD%97%E4%BD%93%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">字体反爬虫</a> <a href="/tags/%E5%AD%97%E7%AC%A6%E9%97%AE%E9%A2%98/" style="font-size: 10px;">字符问题</a> <a href="/tags/%E5%AD%A6%E4%B9%A0%E6%96%B9%E6%B3%95/" style="font-size: 10px;">学习方法</a> <a href="/tags/%E5%AE%89%E5%8D%93/" style="font-size: 10px;">安卓</a> <a href="/tags/%E5%AE%9E%E7%94%A8/" style="font-size: 10px;">实用</a> <a href="/tags/%E5%B0%81%E9%9D%A2/" style="font-size: 10px;">封面</a> <a href="/tags/%E5%B4%94%E5%BA%86%E6%89%8D/" style="font-size: 18.75px;">崔庆才</a> <a href="/tags/%E5%B7%A5%E5%85%B7/" style="font-size: 12.5px;">工具</a> <a href="/tags/%E5%BC%80%E5%8F%91%E5%B7%A5%E5%85%B7/" style="font-size: 10px;">开发工具</a> <a href="/tags/%E5%BE%AE%E8%BD%AF/" style="font-size: 10px;">微软</a> <a href="/tags/%E6%80%9D%E8%80%83/" style="font-size: 10px;">思考</a> <a href="/tags/%E6%89%8B%E6%9C%BA%E8%AE%BF%E9%97%AE/" style="font-size: 10px;">手机访问</a> <a href="/tags/%E6%95%99%E7%A8%8B/" style="font-size: 10px;">教程</a> <a href="/tags/%E6%95%99%E8%82%B2/" style="font-size: 10px;">教育</a> <a href="/tags/%E6%96%B0%E4%B9%A6/" style="font-size: 12.5px;">新书</a> <a href="/tags/%E6%96%B9%E6%B3%95%E8%AE%BA/" style="font-size: 10px;">方法论</a> <a href="/tags/%E6%97%85%E6%B8%B8/" style="font-size: 10px;">旅游</a> <a href="/tags/%E6%97%A5%E5%BF%97/" style="font-size: 10px;">日志</a> <a href="/tags/%E6%9A%97%E6%97%B6%E9%97%B4/" style="font-size: 10px;">暗时间</a> <a href="/tags/%E6%9D%9C%E5%85%B0%E7%89%B9/" style="font-size: 11.25px;">杜兰特</a> <a href="/tags/%E6%A1%8C%E9%9D%A2/" style="font-size: 10px;">桌面</a> <a href="/tags/%E6%AD%8C%E5%8D%95/" style="font-size: 10px;">歌单</a> <a href="/tags/%E6%B1%9F%E5%8D%97/" style="font-size: 10px;">江南</a> <a href="/tags/%E6%B8%B8%E6%88%8F/" style="font-size: 10px;">游戏</a> <a href="/tags/%E7%84%A6%E8%99%91/" style="font-size: 10px;">焦虑</a> <a href="/tags/%E7%88%AC%E8%99%AB/" style="font-size: 16.25px;">爬虫</a> <a href="/tags/%E7%88%AC%E8%99%AB%E4%B9%A6%E7%B1%8D/" style="font-size: 11.25px;">爬虫书籍</a> <a href="/tags/%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F/" style="font-size: 10px;">环境变量</a> <a href="/tags/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/" style="font-size: 10px;">生活笔记</a> <a href="/tags/%E7%99%BB%E5%BD%95/" style="font-size: 10px;">登录</a> <a href="/tags/%E7%9F%A5%E4%B9%8E/" style="font-size: 10px;">知乎</a> <a href="/tags/%E7%9F%AD%E4%BF%A1/" style="font-size: 10px;">短信</a> <a href="/tags/%E7%9F%AD%E4%BF%A1%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">短信验证码</a> <a href="/tags/%E7%AC%94%E8%AE%B0%E8%BD%AF%E4%BB%B6/" style="font-size: 10px;">笔记软件</a> <a href="/tags/%E7%AF%AE%E7%BD%91/" style="font-size: 10px;">篮网</a> <a href="/tags/%E7%BA%B8%E5%BC%A0/" style="font-size: 10px;">纸张</a> <a href="/tags/%E7%BB%84%E4%BB%B6/" style="font-size: 10px;">组件</a> <a href="/tags/%E7%BD%91%E7%AB%99/" style="font-size: 10px;">网站</a> <a href="/tags/%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB/" style="font-size: 11.25px;">网络爬虫</a> <a href="/tags/%E7%BE%8E%E5%AD%A6/" style="font-size: 10px;">美学</a> <a href="/tags/%E8%82%89%E5%A4%B9%E9%A6%8D/" style="font-size: 10px;">肉夹馍</a> <a href="/tags/%E8%85%BE%E8%AE%AF%E4%BA%91/" style="font-size: 10px;">腾讯云</a> <a href="/tags/%E8%87%AA%E5%BE%8B/" style="font-size: 10px;">自律</a> <a href="/tags/%E8%A5%BF%E5%B0%91%E7%88%B7/" style="font-size: 10px;">西少爷</a> <a href="/tags/%E8%A7%86%E9%A2%91/" style="font-size: 10px;">视频</a> <a href="/tags/%E8%B0%B7%E6%AD%8C%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">谷歌验证码</a> <a href="/tags/%E8%BF%90%E8%90%A5/" style="font-size: 10px;">运营</a> <a href="/tags/%E8%BF%9C%E7%A8%8B/" style="font-size: 10px;">远程</a> <a href="/tags/%E9%80%86%E5%90%91/" style="font-size: 10px;">逆向</a> <a href="/tags/%E9%85%8D%E7%BD%AE/" style="font-size: 10px;">配置</a> <a href="/tags/%E9%87%8D%E8%A3%85/" style="font-size: 10px;">重装</a> <a href="/tags/%E9%98%BF%E6%9D%9C/" style="font-size: 10px;">阿杜</a> <a href="/tags/%E9%9D%99%E8%A7%85/" style="font-size: 17.5px;">静觅</a> <a href="/tags/%E9%A2%A0%E8%A6%86/" style="font-size: 10px;">颠覆</a> <a href="/tags/%E9%A3%9E%E4%BF%A1/" style="font-size: 10px;">飞信</a> <a href="/tags/%E9%B8%BF%E8%92%99/" style="font-size: 10px;">鸿蒙</a>
              </div>
              <script>
                const tagsColors = ['#00a67c', '#5cb85c', '#d9534f', '#567e95', '#b37333', '#f4843d', '#15a287']
                const tagsElements = document.querySelectorAll('.sidebar-panel-tags .content a')
                tagsElements.forEach((item) =>
                {
                  item.style.backgroundColor = tagsColors[Math.floor(Math.random() * tagsColors.length)]
                })

              </script>
            </div>
            <div class="sidebar-panel sidebar-panel-categories sidebar-panel-active">
              <h4 class="name"> 分类 </h4>
              <div class="content">
                <ul class="category-list">
                  <li class="category-list-item"><a class="category-list-link" href="/categories/C-C/">C/C++</a><span class="category-list-count">23</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/HTML/">HTML</a><span class="category-list-count">14</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Java/">Java</a><span class="category-list-count">5</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/JavaScript/">JavaScript</a><span class="category-list-count">26</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Linux/">Linux</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Markdown/">Markdown</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Net/">Net</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Other/">Other</a><span class="category-list-count">39</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/PHP/">PHP</a><span class="category-list-count">27</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Paper/">Paper</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Python/">Python</a><span class="category-list-count">261</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/TypeScript/">TypeScript</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E5%B1%95%E7%A4%BA/">个人展示</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E6%97%A5%E8%AE%B0/">个人日记</a><span class="category-list-count">9</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E8%AE%B0%E5%BD%95/">个人记录</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E9%9A%8F%E7%AC%94/">个人随笔</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AE/">安装配置</a><span class="category-list-count">59</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%8A%80%E6%9C%AF%E6%9D%82%E8%B0%88/">技术杂谈</a><span class="category-list-count">88</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%9C%AA%E5%88%86%E7%B1%BB/">未分类</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/">生活笔记</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%A6%8F%E5%88%A9%E4%B8%93%E5%8C%BA/">福利专区</a><span class="category-list-count">6</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E8%81%8C%E4%BD%8D%E6%8E%A8%E8%8D%90/">职位推荐</a><span class="category-list-count">2</span></li>
                </ul>
              </div>
            </div>
            <div class="sidebar-panel sidebar-panel-friends sidebar-panel-active">
              <h4 class="name"> 友情链接 </h4>
              <ul class="friends">
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/j2dub.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.findhao.net/" target="_blank" rel="noopener">FindHao</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ou6mm.jpg">
                  </span>
                  <span class="link">
                    <a href="https://diygod.me/" target="_blank" rel="noopener">DIYgod</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/6apxu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.51dev.com/" target="_blank" rel="noopener">IT技术社区</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.jankl.com/img/titleshu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.jankl.com/" target="_blank" rel="noopener">liberalist</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/bqlbs.png">
                  </span>
                  <span class="link">
                    <a href="http://www.urselect.com/" target="_blank" rel="noopener">优社电商</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8s88c.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yuanrenxue.com/" target="_blank" rel="noopener">猿人学</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2wgg5.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yunlifang.cn/" target="_blank" rel="noopener">云立方</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/shwr6.png">
                  </span>
                  <span class="link">
                    <a href="http://lanbing510.info/" target="_blank" rel="noopener">冰蓝</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/blvoh.jpg">
                  </span>
                  <span class="link">
                    <a href="https://lengyue.me/" target="_blank" rel="noopener">冷月</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="http://qianxunclub.com/favicon.png">
                  </span>
                  <span class="link">
                    <a href="http://qianxunclub.com/" target="_blank" rel="noopener">千寻啊千寻</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/0044u.jpg">
                  </span>
                  <span class="link">
                    <a href="http://kodcloud.com/" target="_blank" rel="noopener">可道云</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ygnpn.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.kunkundashen.cn/" target="_blank" rel="noopener">坤坤大神</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/22uv1.png">
                  </span>
                  <span class="link">
                    <a href="http://www.cenchong.com/" target="_blank" rel="noopener">岑冲博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ev9kl.png">
                  </span>
                  <span class="link">
                    <a href="http://www.zxiaoji.com/" target="_blank" rel="noopener">张小鸡</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.503error.com/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.503error.com/" target="_blank" rel="noopener">张志明个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/x714o.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.hubwiz.com/" target="_blank" rel="noopener">汇智网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/129d8.png">
                  </span>
                  <span class="link">
                    <a href="https://www.bysocket.com/" target="_blank" rel="noopener">泥瓦匠BYSocket</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.xiongge.club/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.xiongge.club/" target="_blank" rel="noopener">熊哥club</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/3w4fe.png">
                  </span>
                  <span class="link">
                    <a href="https://zerlong.com/" target="_blank" rel="noopener">知语</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/44hxf.png">
                  </span>
                  <span class="link">
                    <a href="http://redstonewill.com/" target="_blank" rel="noopener">红色石头</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8g1fk.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.laodong.me/" target="_blank" rel="noopener">老董博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/wkaus.jpg">
                  </span>
                  <span class="link">
                    <a href="https://zhaoshuai.me/" target="_blank" rel="noopener">碎念</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/pgo0r.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.chenwenguan.com/" target="_blank" rel="noopener">陈文管的博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/kk82a.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.lxlinux.net/" target="_blank" rel="noopener">良许Linux教程网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/lj0t2.jpg">
                  </span>
                  <span class="link">
                    <a href="https://tanqingbo.cn/" target="_blank" rel="noopener">IT码农</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/i8cdr.png">
                  </span>
                  <span class="link">
                    <a href="https://junyiseo.com/" target="_blank" rel="noopener">均益个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/chwv2.png">
                  </span>
                  <span class="link">
                    <a href="https://brucedone.com/" target="_blank" rel="noopener">大鱼的鱼塘</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2y43o.png">
                  </span>
                  <span class="link">
                    <a href="http://bbs.nightteam.cn/" target="_blank" rel="noopener">夜幕爬虫安全论坛</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/zvc3w.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.weishidong.com/" target="_blank" rel="noopener">韦世东的技术专栏</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ebudy.jpg">
                  </span>
                  <span class="link">
                    <a href="https://chuanjiabing.com/" target="_blank" rel="noopener">穿甲兵技术社区</a>
                  </span>
                </li>
              </ul>
            </div>
          </div>
        </aside>
        <div id="sidebar-dimmer"></div>
      </div>
    </main>
    <footer class="footer">
      <div class="footer-inner">
        <div class="copyright"> &copy; <span itemprop="copyrightYear">2021</span>
          <span class="with-love">
            <i class="fa fa-heart"></i>
          </span>
          <span class="author" itemprop="copyrightHolder">崔庆才丨静觅</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-chart-area"></i>
          </span>
          <span title="站点总字数">2.6m</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-coffee"></i>
          </span>
          <span title="站点阅读时长">39:54</span>
        </div>
        <div class="powered-by">由 <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://pisces.theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Pisces</a> 强力驱动 </div>
        <div class="beian"><a href="https://beian.miit.gov.cn/" rel="noopener" target="_blank">京ICP备18015597号-1 </a>
        </div>
        <script>
          (function ()
          {
            function leancloudSelector(url)
            {
              url = encodeURI(url);
              return document.getElementById(url).querySelector('.leancloud-visitors-count');
            }

            function addCount(Counter)
            {
              var visitors = document.querySelector('.leancloud_visitors');
              var url = decodeURI(visitors.id);
              var title = visitors.dataset.flagTitle;
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                if (results.length > 0)
                {
                  var counter = results[0];
                  leancloudSelector(url).innerText = counter.time + 1;
                  Counter('put', '/classes/Counter/' + counter.objectId,
                  {
                    time:
                    {
                      '__op': 'Increment',
                      'amount': 1
                    }
                  }).catch(error =>
                  {
                    console.error('Failed to save visitor count', error);
                  });
                }
                else
                {
                  Counter('post', '/classes/Counter',
                  {
                    title,
                    url,
                    time: 1
                  }).then(response => response.json()).then(() =>
                  {
                    leancloudSelector(url).innerText = 1;
                  }).catch(error =>
                  {
                    console.error('Failed to create', error);
                  });
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }

            function showTime(Counter)
            {
              var visitors = document.querySelectorAll('.leancloud_visitors');
              var entries = [...visitors].map(element =>
              {
                return decodeURI(element.id);
              });
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url:
                {
                  '$in': entries
                }
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                for (let url of entries)
                {
                  let target = results.find(item => item.url === url);
                  leancloudSelector(url).innerText = target ? target.time : 0;
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }
            let
            {
              app_id,
              app_key,
              server_url
            } = {
              "enable": true,
              "app_id": "6X5dRQ0pnPWJgYy8SXOg0uID-gzGzoHsz",
              "app_key": "ziLDVEy73ne5HtFTiGstzHMS",
              "server_url": "https://6x5drq0p.lc-cn-n1-shared.com",
              "security": false
            };

            function fetchData(api_server)
            {
              var Counter = (method, url, data) =>
              {
                return fetch(`${api_server}/1.1${url}`,
                {
                  method,
                  headers:
                  {
                    'X-LC-Id': app_id,
                    'X-LC-Key': app_key,
                    'Content-Type': 'application/json',
                  },
                  body: JSON.stringify(data)
                });
              };
              if (CONFIG.page.isPost)
              {
                if (CONFIG.hostname !== location.hostname) return;
                addCount(Counter);
              }
              else if (document.querySelectorAll('.post-title-link').length >= 1)
              {
                showTime(Counter);
              }
            }
            let api_server = app_id.slice(-9) !== '-MdYXbMMI' ? server_url : `https://${app_id.slice(0, 8).toLowerCase()}.api.lncldglobal.com`;
            if (api_server)
            {
              fetchData(api_server);
            }
            else
            {
              fetch('https://app-router.leancloud.cn/2/route?appId=' + app_id).then(response => response.json()).then((
              {
                api_server
              }) =>
              {
                fetchData('https://' + api_server);
              });
            }
          })();

        </script>
      </div>
      <div class="footer-stat">
        <span id="cnzz_stat_icon_1279355174"></span>
        <script type="text/javascript">
          document.write(unescape("%3Cspan id='cnzz_stat_icon_1279355174'%3E%3C/span%3E%3Cscript src='https://v1.cnzz.com/z_stat.php%3Fid%3D1279355174%26online%3D1%26show%3Dline' type='text/javascript'%3E%3C/script%3E"));

        </script>
      </div>
    </footer>
  </div>
  <script src="//cdn.jsdelivr.net/npm/animejs@3.2.1/lib/anime.min.js"></script>
  <script src="//cdn.jsdelivr.net/npm/pangu@4/dist/browser/pangu.min.js"></script>
  <script src="/js/utils.js"></script>
  <script src="/.js"></script>
  <script src="/js/schemes/pisces.js"></script>
  <script src="/.js"></script>
  <script src="/js/next-boot.js"></script>
  <script src="/.js"></script>
  <script>
    (function ()
    {
      var canonicalURL, curProtocol;
      //Get the <link> tag
      var x = document.getElementsByTagName("link");
      //Find the last canonical URL
      if (x.length > 0)
      {
        for (i = 0; i < x.length; i++)
        {
          if (x[i].rel.toLowerCase() == 'canonical' && x[i].href)
          {
            canonicalURL = x[i].href;
          }
        }
      }
      //Get protocol
      if (!canonicalURL)
      {
        curProtocol = window.location.protocol.split(':')[0];
      }
      else
      {
        curProtocol = canonicalURL.split(':')[0];
      }
      //Get current URL if the canonical URL does not exist
      if (!canonicalURL) canonicalURL = window.location.href;
      //Assign script content. Replace current URL with the canonical URL
      ! function ()
      {
        var e = /([http|https]:\/\/[a-zA-Z0-9\_\.]+\.baidu\.com)/gi,
          r = canonicalURL,
          t = document.referrer;
        if (!e.test(r))
        {
          var n = (String(curProtocol).toLowerCase() === 'https') ? "https://sp0.baidu.com/9_Q4simg2RQJ8t7jm9iCKT-xh_/s.gif" : "//api.share.baidu.com/s.gif";
          t ? (n += "?r=" + encodeURIComponent(document.referrer), r && (n += "&l=" + r)) : r && (n += "?l=" + r);
          var i = new Image;
          i.src = n
        }
      }(window);
    })();

  </script>
  <script src="/js/local-search.js"></script>
  <script src="/.js"></script>
  <link rel="stylesheet" href="//cdn.jsdelivr.net/npm/gitalk@1/dist/gitalk.min.css">
  <script>
    NexT.utils.loadComments(document.querySelector('#gitalk-container'), () =>
    {
      NexT.utils.getScript('//cdn.jsdelivr.net/npm/gitalk@1/dist/gitalk.min.js', () =>
      {
        var gitalk = new Gitalk(
        {
          clientID: '4c86ce1d7c4fbb3b277c',
          clientSecret: '4927beb0f90e2c07e66c99d9d2529cf3eb8ac8e4',
          repo: 'Blog',
          owner: 'germey',
          admin: ['germey'],
          id: 'a68165ffa572cec4ee9b123db085bb67',
          language: 'zh-CN',
          distractionFreeMode: true
        });
        gitalk.render('gitalk-container');
      }, window.Gitalk);
    });

  </script>
</body>

</html>
